const _ = require('lodash')

const Request = require('./request')


module.exports = class Spider {
    constructor(crawler) {
        this.crawler = crawler
        this.logger = require('./singleton').logger.child({'label': 'spider'})
    }

    allow(request) {
        return true
    }

    startUrls() {
        return []
    }

    startRequests() {
        return _.map(this.startUrls(), url => {
            if (url instanceof Request) {
                return url
            } else if (_.isString(url)) {
                return new Request(url, {})
            } else if (_.isArray(url)) {
                const [arg1, arg2] = url
                return new Request(arg1, arg2)
            } else {
                this.logger.error(`invalid url: ${url}`)
            }
        })
    }

    parse(response, {request, collect, follow}) {
        throw new Error(`${this.constructor.name}.parse is not implemented`)
    }
}